from twitter import *
import pickle
import os
from auto_oauth_dance import auto_oauth_dance
from etc import CONSUMER_KEY, CONSUMER_SECRET, TOKEN_CHOOSE, TWITTER_USERS
from dbtools.mongo import get_mongodb_twitter_connection
from dbtools.mysql import get_connection
import time
from etc.logger import get_logger
# from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
import threading
import urllib
import http

# r = t.followers.list(screen_name='std-risk-calculator')
# res = t.users.search(q='ZygoCorp')
# res = t.application.rate_limit_status()

# if len(res)>0:
#     for each in res:
#         res['crunchbase']={'',}
# r = json.dumps(res)
# print(r)
# print(res.rate_limit_remaining)
# print(res.rate_limit_reset)
# print(res.rate_limit_limit)
# print(time.time())
twitter_client = []
connection = get_connection()
mongodb_account_connection = get_mongodb_twitter_connection('account')
mongodb_successful_connection = get_mongodb_twitter_connection('successful_account')
logger = get_logger('twitter_spider')


def get_twitter_client():
    for each in TWITTER_USERS:
        oauth_token = ''
        oauth_token_secret = ''
        user_name = each['user']
        pwd = each['pwd']
        if os.path.exists('token_temp/%s_token'%user_name) and TOKEN_CHOOSE == 2:
            with open('token_temp/%s_token'%user_name, 'rb')as f:
                oauth_token = pickle.load(f)
                oauth_token_secret = pickle.load(f)
        if not oauth_token:
            try:
                oauth_token,oauth_token_secret = auto_oauth_dance('ider_zh',CONSUMER_KEY,CONSUMER_SECRET, user_name, pwd)
                with open('token_temp/%s_token'%user_name, 'wb')as f:
                    pickle.dump(oauth_token, f)
                    pickle.dump(oauth_token_secret, f)
            except Exception as e:
                logger.exception(e)
                print(user_name)
                exit()
        t = Twitter(auth=OAuth(oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET),retry=True)
        twitter_client.append(t)


def work(entry):
    uuid = entry['uuid']
    twitter_name = entry['twitter_url'].strip('/').split('/')[-1]
    tread_id = int(threading.current_thread().name.split('-')[1])-1
    t = twitter_client[tread_id]
    while True:
        try:
            res = t.users.search(q=twitter_name,_timeout=10)
            if len(res) < 1:
                logger.error(twitter_name)
            else:
                result = mongodb_account_connection.insert_many(res)
                mongodb_successful_connection.insert({'uuid': uuid, 'twitters': result.inserted_ids})
                print('%s : %s'%(tread_id,twitter_name))

            break
        except TwitterHTTPError as e:
            try:
                res = t.application.rate_limit_status()
                t = res['resources']['users']['/users/search']['reset']-time.time()+1
                print('%s :sleep %s s'%(twitter_name, t))
                time.sleep(t)
            except Exception as e:
                print(e)
                print('%s err'%tread_id)
                time.sleep(60)
        except (urllib.error.URLError, http.client.RemoteDisconnected, OSError) as e:
            print('%s : %s'%(e, tread_id))
            time.sleep(10)
        except Exception as e:
            logger.exception(e)
            logger.error(entry['twitter_url'])
            break


def main():
    get_twitter_client()
    successful_list = []

    for each in mongodb_successful_connection.find({}):
        successful_list.append(each['uuid'])
    print(len(successful_list))

    pool_len = len(twitter_client)
    pool = ThreadPool(pool_len)

    work_list = []
    with connection.cursor() as cursor:
        sql = "SELECT `uuid`,`twitter_url` FROM `organizationsummary` WHERE `twitter_url` IS NOT NULL"
        cursor.execute(sql)
        result = cursor.fetchall()
        for each in result:
            if each['uuid'] in successful_list:
                continue
            work_list.append(each)
    print(len(work_list))
    pool.map(work, work_list)


if __name__ == '__main__':
    main()